package indexer;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Vector;

import util.CommonFactory;
import util.Constants;

import algorithm.text.wordsegment.WordSegmentor;
import algorithm.text.wordsegment.WordSegmentor.TokenUnit;

/**
 * 读入文件，并且分词。
 * @author aleck
 *
 */
public class TextFileSegmentor {
	private static WordSegmentor ws = CommonFactory.getWordSegmentor();
	
	public final String fileName;
	private String content; 
	
	public TextFileSegmentor(String fileName) {
		this.fileName = fileName;
		readContent();
	}
	
	private void readContent() {
		BufferedReader br;
		try {
			br = new BufferedReader(
					new InputStreamReader(new FileInputStream(fileName),"utf-8"));
			//the above sentence is modified by shitgao, he added utf8 things
			StringBuilder sb = new StringBuilder();
			String line;
			while ((line = br.readLine()) != null) {
				// ignore useless lines
				//changed by shitgao for simpilification
				
				if (line.length() == 0 || line.startsWith(":") || 
						line.startsWith("发信站: 水木社区") || 
						(line.startsWith("【") && line.endsWith("】")))
					continue;
				if (line.startsWith("发信人: ")) {
					// author
					int pos = line.indexOf(',');
					line = line.substring(4, pos);
				} else if (line.startsWith("标  题: ")) {
					// title
					line = line.substring(4);
				}
				//above is changed by shitgao
				sb.append(line);
				sb.append('\n');
			}
			br.close();
			content = sb.toString();
		} catch (FileNotFoundException e) {
			System.err.println("File " + fileName + " not found!");
			e.printStackTrace();
		} catch (IOException e) {
			System.err.println("Error opening file " + fileName);
			e.printStackTrace();
		}
	}

	/**
	 * return the content of file
	 * @return
	 */
	public String getFileContent() {
		return content;
	}
	
	/**
	 * do word segmentation
	 * @return
	 */
	public List<TokenUnit> getWords() {
		return ws.segmentToToken(content, 0, content.length(), WordSegmentor.ALGORITHM_DYNAMIC);
	}
	
	//added by shitgao, return Vector<WordUnit> for the convenience of ReverseIndex
	public Vector<WordUnit> turnToWordUnits  () {
		List<String> stringList=ws.segmentToWords(content, 0, content.length(), WordSegmentor.ALGORITHM_DYNAMIC);
		Vector<WordUnit> wordVec=new Vector<WordUnit>();
		String str;
		WordUnit wu;
		for (int i=0;i<stringList.size();i++){
			str=stringList.get(i);
			wu=new WordUnit();
			/*try{
				String str= new String(tu.buffer.substring(tu.beginIndex,tu.endIndex-1).getBytes("UTF-8"),"UNICODE");
				wu.aString=str;
			}catch (Exception e){
				System.out.println("I can not turn utf-8 into unicode in this situation" + 
						tu.buffer);
			}*/
			wu.aString=str;
			wu.offSet=i;
			wordVec.add(wu);
		}
		return wordVec;
	}
}